<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <meta name="author" content="PG-Strom Development Team">
  <link rel="shortcut icon" href="../img/favicon.ico">
  <title>Gstore_fdw - PG-Strom Manual</title>
  <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>

  <link rel="stylesheet" href="../css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
  <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
  <link href="//fonts.googleapis.com/earlyaccess/notosansjp.css" rel="stylesheet">
  <link href="//fonts.googleapis.com/css?family=Open+Sans:600,800" rel="stylesheet">
  <link href="../custom.css" rel="stylesheet">
  
  <script>
    // Current page data
    var mkdocs_page_name = "Gstore_fdw";
    var mkdocs_page_input_path = "gstore_fdw.md";
    var mkdocs_page_url = null;
  </script>
  
  <script src="../js/jquery-2.1.1.min.js" defer></script>
  <script src="../js/modernizr-2.8.3.min.js" defer></script>
  <script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
  <script>hljs.initHighlightingOnLoad();</script> 
  
</head>

<body class="wy-body-for-nav" role="document">

  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
      <div class="wy-side-nav-search">
        <a href=".." class="icon icon-home"> PG-Strom Manual</a>
        <div role="search">
  <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
  </form>

  [<a href="../ja" style="color: #cccccc">Japanese</a> | <strong>English</strong>]

</div>
      </div>

      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
	<ul class="current">
	  
          
            <li class="toctree-l1">
		
    <a class="" href="..">Home</a>
	    </li>
          
            <li class="toctree-l1">
		
    <a class="" href="../install/">Install</a>
	    </li>
          
            <li class="toctree-l1">
		
    <span class="caption-text">Tutorial</span>
    <ul class="subnav">
                <li class="">
                    
    <a class="" href="../operations/">Basic Operations</a>
                </li>
                <li class="">
                    
    <a class="" href="../sys_admin/">System Administration</a>
                </li>
                <li class="">
                    
    <a class="" href="../brin/">Index Support</a>
                </li>
                <li class="">
                    
    <a class="" href="../partition/">Partitioning</a>
                </li>
                <li class="">
                    
    <a class="" href="../troubles/">Trouble Shooting</a>
                </li>
    </ul>
	    </li>
          
            <li class="toctree-l1">
		
    <span class="caption-text">Advanced Features</span>
    <ul class="subnav">
                <li class="">
                    
    <a class="" href="../ssd2gpu/">SSD2GPU Direct SQL</a>
                </li>
                <li class="">
                    
    <a class="" href="../arrow_fdw/">Arrow_fdw</a>
                </li>
                <li class=" current">
                    
    <a class="current" href="./">Gstore_fdw</a>
    <ul class="subnav">
            
    <li class="toctree-l3"><a href="#overview">Overview</a></li>
    

    <li class="toctree-l3"><a href="#setup">Setup</a></li>
    

    <li class="toctree-l3"><a href="#operations">Operations</a></li>
    
        <ul>
        
            <li><a class="toctree-l4" href="#loading-data">Loading data</a></li>
        
            <li><a class="toctree-l4" href="#checking-the-memory-consumption">Checking the memory consumption</a></li>
        
            <li><a class="toctree-l4" href="#internal-data-format">Internal Data Format</a></li>
        
        </ul>
    

    <li class="toctree-l3"><a href="#inter-process-data-collaboration">Inter-process Data Collaboration</a></li>
    
        <ul>
        
            <li><a class="toctree-l4" href="#sql-functions-to">SQL Functions to</a></li>
        
        </ul>
    

    </ul>
                </li>
                <li class="">
                    
    <a class="" href="../plcuda/">PL/CUDA</a>
                </li>
    </ul>
	    </li>
          
            <li class="toctree-l1">
		
    <span class="caption-text">References</span>
    <ul class="subnav">
                <li class="">
                    
    <a class="" href="../ref_types/">Data Types</a>
                </li>
                <li class="">
                    
    <a class="" href="../ref_devfuncs/">Functions and Operators</a>
                </li>
                <li class="">
                    
    <a class="" href="../ref_sqlfuncs/">SQL Objects</a>
                </li>
                <li class="">
                    
    <a class="" href="../ref_params/">GUC Parameters</a>
                </li>
    </ul>
	    </li>
          
            <li class="toctree-l1">
		
    <a class="" href="../release_note/">Release Note</a>
	    </li>
          
        </ul>
      </div>
      &nbsp;
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
        <a href="..">PG-Strom Manual</a>
      </nav>

      
      <div class="wy-nav-content">
        <div class="rst-content">
          <div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
    <li><a href="..">Docs</a> &raquo;</li>
    
      
        
          <li>Advanced Features &raquo;</li>
        
      
    
    <li>Gstore_fdw</li>
    <li class="wy-breadcrumbs-aside">
      
    </li>
  </ul>
  <hr/>
</div>
          <div role="main">
            <div class="section">
              
                <h1>GPU Memory Store(gstore_fdw)</h1>

<h1 id="overview">Overview</h1>
<p>Usually, PG-Strom uses GPU device memory for temporary purpose only. It allocates a certain amount of device memory needed for query execution, then transfers data blocks and launch GPU kernel to process SQL workloads. Once GPU kernel gets finished, these device memory regison shall be released soon, to re-allocate unused device memory for other workloads.</p>
<p>This design allows concurrent multiple session or scan workloads on the tables larger than GPU device memory. It may not be optimal depending on circumstances.</p>
<p>A typical example is, repeated calculation under various conditions for data with a scale large enough to fit in the GPU device memory, not so large. This applies to workloads such as machine-learning, pattern matching or similarity search.</p>
<p>For modern GPUs, it is not so difficult to process a few gigabytes data on memory at most, but it is a costly process to setup data to be loaded onto GPU device memory and transfer them.</p>
<p>In addition, since variable length data in PostgreSQL has size limitation up to 1GB, it restricts the data format when it is givrn as an argument of PL/CUDA function, even if the data size itself is sufficient in the GPU device memory.</p>
<p>GPU memory store (gstore_fdw) is a feature to preserve GPU device memory and to load data to the memory preliminary.
It makes unnecessary to setup arguments and load for each invocation of PL/CUDA function, and eliminates 1GB limitation of variable length data because it allows GPU device memory allocation up to the capacity.</p>
<p>As literal, gstore_fdw is implemented using foreign-data-wrapper of PostgreSQL.
You can modify the data structure on GPU device memory using <code>INSERT</code>, <code>UPDATE</code> or <code>DELETE</code> commands on the foreign table managed by gstore_fdw. In the similar way, you can also read the data using <code>SELECT</code> command.</p>
<p>PL/CUDA function can reference the data stored onto GPU device memory through the foreign table.
Right now, GPU programs which is transparently generated from SQL statement cannot reference this device memory region, however, we plan to enhance the feature in the future release.</p>
<p><img alt="GPU memory store" src="../img/gstore_fdw-overview.png" /></p>
<h1 id="setup">Setup</h1>
<p>Usually it takes the 3 steps below to create a foreign table.</p>
<ul>
<li>Define a foreign-data-wrapper using <code>CREATE FOREIGN DATA WRAPPER</code> command</li>
<li>Define a foreign server using <code>CREATE SERVER</code> command</li>
<li>Define a foreign table using <code>CREATE FOREIGN TABLE</code> command</li>
</ul>
<p>The first 2 steps above are included in the <code>CREATE EXTENSION pg_strom</code> command. All you need to run individually is <code>CREATE FOREIGN TABLE</code> command last.</p>
<pre><code>CREATE FOREIGN TABLE ft (
    id int,
    signature smallint[] OPTIONS (compression 'pglz')
)
SERVER gstore_fdw OPTIONS(pinning '0', format 'pgstrom');
</code></pre>

<p>You can specify some options on creation of foreign table using <code>CREATE FOREIGN TABLE</code> command.</p>
<p><code>SERVER gstore_fdw</code> is a mandatory option. It indicates the new foreign table is managed by gstore_fdw.</p>
<p>The options below are supported in the <code>OPTIONS</code> clause.</p>
<table>
<thead>
<tr>
<th align="center">name</th>
<th align="center">target</th>
<th align="left">description</th>
</tr>
</thead>
<tbody>
<tr>
<td align="center"><code>pinning</code></td>
<td align="center">table</td>
<td align="left">Specifies device number of the GPU where device memory is preserved.</td>
</tr>
<tr>
<td align="center"><code>format</code></td>
<td align="center">table</td>
<td align="left">Specifies the internal data format on GPU device memory. Default is <code>pgstrom</code></td>
</tr>
<tr>
<td align="center"><code>compression</code></td>
<td align="center">column</td>
<td align="left">Specifies whether variable length data is compressed, or not. Default is uncompressed.</td>
</tr>
</tbody>
</table>
<p>Right now, only <code>pgstrom</code> is supported for <code>format</code> option. It is an original data format of PG-Strom to store structured data of PostgreSQL in columnar format.
In most cases, no need to pay attention to internal data format on writing / reading GPU data store using SQL. On the other hands, you need to consider when you program PL/CUDA function or share the GPU device memory with external applications using IPC handle.</p>
<p>Right now, only <code>pglz</code> is supported for <code>compression</code> option. This compression logic adopts an identical data format and algorithm used by PostgreSQL to compress variable length data larger than its threshold.
It can be decompressed by GPU internal function <code>pglz_decompress()</code> from PL/CUDA function. Due to the characteristics of the compression algorithm, it is valuable to represent sparse matrix that is mostly zero.</p>
<h1 id="operations">Operations</h1>
<h2 id="loading-data">Loading data</h2>
<p>Like normal tables, you can write GPU device memory on behalf of the foreign table using <code>INSERT</code>, <code>UPDATE</code> and <code>DELETE</code> command.</p>
<p>Note that gstore_fdw acquires <code>SHARE UPDATE EXCLUSIVE</code> lock on the beginning of these commands. It means only single transaction can update the gstore_fdw foreign table at a certain point.
It is a trade-off. We don't need to check visibility per record when PL/CUDA function references gstore_fdw foreign table.</p>
<p>Any contents written to the gstore_fdw foreign table is not visible to other sessions until transaction getting committed, like regular tables.
This is a significant feature to ensure atomicity of transaction, however, it also means the older revision of gstore_fdw foreign table contents must be kept on the GPU device memory until any concurrent transaction which may reference the older revision gets committed or aborted.</p>
<p>So, even though you can run <code>INSERT</code>, <code>UPDATE</code> or <code>DELETE</code> commands as if it is regular tables, you should avoidto update several rows then commit transaction many times. Basically, <code>INSERT</code> of massive rows at once (bulk loading) is recommended.</p>
<p>Unlike regular tables, contents of the gstore_fdw foreign table is vollatile. So, it is very easy to loose contents of the gstore_fdw foreign table by power-down or PostgreSQL restart. So, what we load onto gstore_fdw foreign table should be reconstructable by other data source.</p>
<h2 id="checking-the-memory-consumption">Checking the memory consumption</h2>
<p>See <code>pgstrom.gstore_fdw_chunk_info</code> system view to see amount of the device memory consumed by gstore_fdw.</p>
<pre><code>postgres=# select * from pgstrom.gstore_fdw_chunk_info ;
 database_oid | table_oid | revision | xmin | xmax | pinning | format  |  rawsize  |  nitems
--------------+-----------+----------+------+------+---------+---------+-----------+----------
        13806 |     26800 |        3 |    2 |    0 |       0 | pgstrom | 660000496 | 15000000
        13806 |     26797 |        2 |    2 |    0 |       0 | pgstrom | 440000496 | 10000000
(2 rows)
</code></pre>

<p>By <code>nvidia-smi</code> command, you can check how much device memory is consumed for each GPU device.
"PG-Strom GPU memory keeper" process actually keeps and manages the device memory area acquired by Gstore_fdw. In this example, 1211MB is preliminary allocated for total of the above rawsize (about 1100MB) and CUDA internal usage.</p>
<pre><code>$ nvidia-smi
Wed Apr  4 15:11:50 2018
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 390.30                 Driver Version: 390.30                    |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla P40           Off  | 00000000:02:00.0 Off |                    0 |
| N/A   39C    P0    52W / 250W |   1221MiB / 22919MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|    0      6885      C   ...bgworker: PG-Strom GPU memory keeper     1211MiB |
+-----------------------------------------------------------------------------+
</code></pre>

<h2 id="internal-data-format">Internal Data Format</h2>
<p>See the notes for details of the internal data format when gstore_fdw write on GPU device memory.</p>
<ul>
<li><a href="https://github.com/heterodb/pg-strom/wiki/301:-Gstore_fdw-internal-format-of-%27pgstrom%27">Detail of the <code>pgstrom</code> format</a></li>
</ul>
<h1 id="inter-process-data-collaboration">Inter-process Data Collaboration</h1>
<p>CUDA provides special APIs <code>cuIpcGetMemHandle()</code> and <code>cuIpcOpenMemHandle()</code>.
The first allows to get a unique identifier of GPU device memory allocated by applications. The other one allows to reference a shared GPU device memory region from other applications. In the other words, it supports something like a shared memory on the host system.</p>
<p>This unique identifier is <code>CUipcMemHandle</code> object; which is simple binary data in 64bytes.
This session introduces SQL functions which exchange GPU device memory with other applications using <code>CUipcMemHandle</code> identifier.</p>
<h2 id="sql-functions-to">SQL Functions to</h2>
<h3 id="gstore_export_ipchandlereggstore">gstore_export_ipchandle(reggstore)</h3>
<p>This function gets <code>CUipcMemHandle</code> identifier of the GPU device memory which is preserved by gstore_fdw foreign table, then returns as a binary data in <code>bytea</code> type.
If foreign table is empty and has no GPU device memory, it returns NULL.</p>
<ul>
<li>1st arg(<em>ftable_oid</em>): OID of the foreign table. Because it is <code>reggstore</code> type, you can specify the foreign table by name string.</li>
<li>result: <code>CUipcMemHandle</code> identifier in the bytea type.</li>
</ul>
<pre><code># select gstore_export_ipchandle('ft');
                                                      gstore_export_ipchandle

------------------------------------------------------------------------------------------------------------------------------------
 \xe057880100000000de3a000000000000904e7909000000000000800900000000000000000000000000020000000000005c000000000000001200d0c10101005c
(1 row)
</code></pre>

<h3 id="lo_import_gpuint-bytea-bigint-bigint-oid0">lo_import_gpu(int, bytea, bigint, bigint, oid=0)</h3>
<p>This function temporary opens the GPU device memory region acquired by external applications, then read this region and writes out as a largeobject of PostgreSQL.
If largeobject already exists, its contents is replaced by the data read from the GPU device memory. It keeps owner and permission configuration. Elsewhere, it creates a new largeobject, then write out the data which is read from GPU device memory.</p>
<ul>
<li>1st arg(<em>device_nr</em>): GPU device number where device memory is acquired</li>
<li>2nd arg(<em>ipc_mhandle</em>): <code>CUipcMemHandle</code> identifier in bytea type</li>
<li>3rd(<em>offset</em>): offset of the head position to read, from the GPU device memory region.</li>
<li>4th(<em>length</em>): size to read in bytes</li>
<li>5th(<em>loid</em>): OID of the largeobject to be written. 0 is assumed, if no valid value is supplied.</li>
<li>result: OID of the written largeobject</li>
</ul>
<h3 id="lo_export_gpuoid-int-bytea-bigint-bigint">lo_export_gpu(oid, int, bytea, bigint, bigint)</h3>
<ul>
<li>1st arg(<em>loid</em>): OID of the largeobject to be read</li>
<li>2nd arg(<em>device_nr</em>): GPU device number where device memory is acquired</li>
<li>3rd arg(<em>ipc_mhandle</em>): <code>CUipcMemHandle</code> identifier in bytea type</li>
<li>4th arg(<em>offset</em>): offset of the head position to write, from the GPU device memory region.</li>
<li>5th arg(<em>length</em>): size to write in bytes</li>
<li>result: Length of bytes actually written. If length of the largeobject is less then <em>length</em>, it may return the value less than <em>length</em>.</li>
</ul>
              
            </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="../plcuda/" class="btn btn-neutral float-right" title="PL/CUDA">Next <span class="icon icon-circle-arrow-right"></span></a>
      
      
        <a href="../arrow_fdw/" class="btn btn-neutral" title="Arrow_fdw"><span class="icon icon-circle-arrow-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <!-- Copyright etc -->
    
  </div>

  Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
      
        </div>
      </div>

    </section>

  </div>

  <div class="rst-versions" role="note" style="cursor: pointer">
    <span class="rst-current-version" data-toggle="rst-current-version">
      
      
        <span><a href="../arrow_fdw/" style="color: #fcfcfc;">&laquo; Previous</a></span>
      
      
        <span style="margin-left: 15px"><a href="../plcuda/" style="color: #fcfcfc">Next &raquo;</a></span>
      
    </span>
</div>
    <script>var base_url = '..';</script>
    <script src="../js/theme.js" defer></script>
      <script src="../search/main.js" defer></script>

</body>
</html>
